# -*- coding: utf-8 -*-
import time
import scrapy
import re
from etherscan_spider.items import BlockinfoItem

class BlockinfoSpider(scrapy.Spider):
    name = 'blockinfo'
    allowed_domains = ['etherscan.io']
    def __init__(self, start_block=None, end_block=None, *args, **kwargs):
        super(BlockinfoSpider,self).__init__(*args, **kwargs)
        self.start_block = start_block
        self.loop_block = self.start_block
        self.end_block = end_block
        self.start_urls = ['https://etherscan.io/block/%s' % self.start_block]

    def get_timestamp(self, str):
        """
        正则提取时间戳数据
        :param str: 原料
        :return: 时间戳
        """
        ret = re.findall("[\w]{3}-[\d]{2}-[\d]+ [\d]+:[\d]+:[\d]+", str)
        time_struct = time.strptime(ret[0], '%b-%d-%Y %H:%M:%S')
        time_stamp = time.mktime(time_struct) + 28800
        return time_stamp

    def get_num(self, str):
        ret = re.findall("[\d]+", str)
        return ret[0]

    def get_num2(self, str):
        ret = re.findall("[\d]+", str)
        num_str = ''
        for i in ret:
            num_str += i
        return num_str

    def get_num3(self, str):
        ret = re.findall("[\d]+", str)
        num_str = ''
        for i in range(len(ret) - 2):
            num_str += ret[i]
        return num_str

    def parse(self, response):
        # 获取具体数
        item = BlockinfoItem()
        item['height'] = self.loop_block  # 区块高度
        item['timestamp'] = self.get_timestamp(response.xpath('//div[@class="col-sm-9 cbs"][2]/text()').extract_first())  # 时间戳
        item['hash'] = response.xpath('//div[@class="col-sm-9 cbs"][4]/text()').extract_first()    # 区块的哈希值
        item['parent_hash'] = response.xpath('//div[@class="col-sm-9 cbs"][5]/a/text()').extract_first() #父区块的哈希值
        item['mined_by'] = response.xpath('//div[@class="col-sm-9 cbs"][7]/a[1]/text()').extract_first()   # 矿工哈希值
        item['difficulty'] = self.get_num2(response.xpath('//div[@class="col-sm-9 cbs"][8]/text()').extract_first())  # 开采难度
        item['total_difficulty'] = self.get_num2(response.xpath('//div[@class="col-sm-9 cbs"][9]/text()').extract_first())    # 开采总难度
        item['size'] = self.get_num(response.xpath('//div[@class="col-sm-9 cbs"][10]/text()').extract_first())   # 区块大小
        item['gas_used'] = self.get_num3(response.xpath('//div[@class="col-sm-9 cbs"][11]/text()').extract_first())   # 燃料消耗量
        item['gas_limit'] = self.get_num2(response.xpath('//div[@class="col-sm-9 cbs"][12]/text()').extract_first())  # 燃料上限
        item['nonce'] = response.xpath('//div[@class="col-sm-9 cbs"][13]/text()').extract_first()  # 随机数
        item['block_reward'] = self.get_num(response.xpath('//div[@class="col-sm-9 cbs"][14]/text()').extract_first())   # 区块奖励
        item['extra_date'] = response.xpath('//div[@class="col-sm-9 cbs"][16]/text()').extract_first() # 额外信息
        transaction_num = response.xpath('//div[@class="col-sm-9 cbs"][3]/a[@title="Click To View Transactions"]/text()').extract_first()
        internal_transaction_num = response.xpath('//div[@class="col-sm-9 cbs"][3]/a[@title="Click To View Internal Transactions"]/text()').extract_first()
        tx_url_plas = response.xpath('//div[@class="col-sm-9 cbs"][3]/a[@title="Click To View Transactions"]/@href').extract_first()
        # 交易和合约内部交易都不存在的情况
        if transaction_num == None and internal_transaction_num == None:
            item['transaction_num'] = 0
            item['internal_transaction_num'] = 0
            item['ether_txfee_avg'] = 0
            item['other_contract_txfee_avg'] = 0
            item['all_txfee_avg'] = 0
            item['contract_persent'] = 0
            yield item
        # 只存在合约内部交易的情况
        elif transaction_num == None and internal_transaction_num != None:
            item['transaction_num'] = 0
            item['internal_transaction_num'] = self.get_num(internal_transaction_num)
            item['ether_txfee_avg'] = 0
            item['other_contract_txfee_avg'] = 0
            item['all_txfee_avg'] = 0
            item['contract_persent'] = 0
            # 输出信息
            yield item
        # 只有交易的情况
        elif transaction_num != None and internal_transaction_num == None:
            item['transaction_num'] = self.get_num(transaction_num)  # 交易数
            item['internal_transaction_num'] = 0
            tx_url = 'https://etherscan.io' + tx_url_plas  # 区块内交易的地址
            total_txfee = 0
            txcount = 0
            total_internal_txfee = 0
            internal_txcount = 0
            # 进一步发送请求获取交易信息
            yield scrapy.Request(tx_url, callback=self.parse_txdetail, meta={'item': item,
                                                                             'total_txfee':total_txfee,
                                                                             'txcount':txcount,
                                                                             'total_internal_txfee':total_internal_txfee,
                                                                             'internal_txcount':internal_txcount})
        # 交易和合约内部交易都存在的情况
        else:
            item['transaction_num'] = self.get_num(transaction_num)  # 交易数
            item['internal_transaction_num'] = self.get_num(internal_transaction_num)  # 合约内部交易数
            tx_url = 'https://etherscan.io' + tx_url_plas
            total_txfee = 0
            txcount = 0
            total_internal_txfee = 0
            internal_txcount = 0
            yield scrapy.Request(tx_url, callback=self.parse_txdetail, meta={'item': item,
                                                                             'total_txfee':total_txfee,
                                                                             'txcount':txcount,
                                                                             'total_internal_txfee':total_internal_txfee,
                                                                             'internal_txcount':internal_txcount})
        # 判断是否获取下一个区块信息
        if int(self.loop_block) < int(self.end_block):
            self.loop_block = str(int(self.loop_block)+1)
            next_url = 'https://etherscan.io/block/%s' % self.loop_block
            yield scrapy.Request(next_url, callback=self.parse)

    # 交易手续费统计
    def parse_txdetail(self, response):
        item = response.meta['item']
        total_txfee = response.meta['total_txfee']
        txcount = response.meta['txcount']
        total_internal_txfee = response.meta['total_internal_txfee']
        internal_txcount = response.meta['internal_txcount']
        # 先分组
        tr_list = response.xpath('//tbody//tr')
        # 遍历列表
        for tr in tr_list:
            # 首先判断是否是失败的交易
            is_failed = tr.xpath('./td[1]/font/@color').extract_first()
            is_contract = tr.xpath('./td[6]/span/i/@rel').extract_first()
            # # 测试
            # item['is_failed'] = is_failed
            # item['is_contract'] = is_contract
            if is_failed == 'red':
                continue
            # 然后判断是否是智能合约
            if is_contract == 'tooltip':
                # 合约的手续费累加，计数加一
                internal_txfee = tr.xpath('./td[8]/font/text()[1]').extract_first() + '.' + tr.xpath('./td[8]/font/text()[2]').extract_first()
                total_internal_txfee += float(internal_txfee)
                internal_txcount += 1
                continue
            # 按以太交易处理
            txfee = tr.xpath('./td[8]/font/text()[1]').extract_first() + '.' + tr.xpath('./td[8]/font/text()[2]').extract_first()
            total_txfee += float(txfee)
            txcount += 1
        # 判断是否需要翻页
        if response.xpath('//div[@class="col-md-6"]/p/span/a[3]/@href').extract_first() != '#':
            # 下一页url
            next_url = 'https://etherscan.io/%s' % response.xpath('//div[@class="col-md-6"]/p/span/a[3]/@href').extract_first()
            yield scrapy.Request(next_url, callback=self.parse_txdetail, meta={'item': item,
                                                                               'total_txfee':total_txfee,
                                                                               'txcount':txcount,
                                                                               'total_internal_txfee':total_internal_txfee,
                                                                               'internal_txcount':internal_txcount})
        # 翻页结束
        else:
            # 计算以太坊手续费均值
            item['ether_txfee_avg'] = total_txfee / txcount
            # 计算其他手续费均值
            item['other_contract_txfee_avg'] = total_internal_txfee / internal_txcount
            # 计算总的手续费均值
            item['all_txfee_avg'] = (total_txfee + total_internal_txfee) / (txcount + internal_txcount)
            # 计算合约占比
            item['contract_persent'] = internal_txcount / (txcount + internal_txcount)
            yield item